#coding:utf-8
import re,hashlib,time,urllib,httplib
from middle.bloomFilterSpider import bloomFilterSpider

class BiyingSiteSpider(bloomFilterSpider):
    '''
bing站内搜索关键字
use case :
a = BiyingSiteSpider("baidu")
a.start("www.scu.edu.cn","谢和平",1)
    '''
    def start(self,site,key,page):
        self.site = site
        site = self.getSiteByUrl(site)
        self.key = key
        key = {'q':"site:("+site+") "+key}
        for i in range(page):
            self.run('http://cn.bing.com/search?'+urllib.urlencode(key)+'&first='+str(i*10+1))
    def extract(self,html):
        # url,pubtime,title,content,searchword,md5
         for sel in html.xpath('//li[@class="b_algo"]'):
            lable = sel.xpath('h2/a')
            timeDiv = sel.xpath('string(div[@class="b_caption"]/div[@class="b_attribution"]/text())')
            if lable and timeDiv:
                lable = lable[0]
                res = {}
                res["url"] = lable.attrib.get("href","")
                if self.existUrl(res["url"]):
                    continue;
                res["pubtime"] = self.convertTime(timeDiv)
                res["site"] = self.site
                res["title"] = self.getInnerText(lable)
                res["content"] = self.opener.openurl(res["url"])
                res["md5"] = hashlib.md5(res["content"]).hexdigest()
                res["searchword"] = self.key
                yield res
    def convertTime(self,s):
        m = re.match(u"(\d+)分",s)
        if m:
            return int(time.time() - int(m.group(1))*60)
        m = re.match(u"(\d+)小时",s)
        if m:
            return int(time.time() - int(m.group(1))*3600)
        m = re.match(u"(\d+)天",s)
        if m:
            return int(time.time() - int(m.group(1))*86400)
        m = re.match("\d+-\d+-\d+",s)
        if m:
            return int(time.mktime(time.strptime(s,"%Y-%m-%d")))
        return "0"
